%% combine tables
clear
% foldername = 'TQ_20120101T000000_20181230T000000\TQ_table';
foldername = 'TQnew2_20120101T000000_20181230T000000\TQ_table';
batchcount = 5110;

%% Setup the Import Options and import the data

% opts = delimitedTextImportOptions("NumVariables", 161);
% 
% % Specify range and delimiter
% opts.DataLines = [2, Inf];
% opts.Delimiter = ",";
% 
% % Specify column names and types
% opts.VariableNames = ["T"     , "stdamp", "meanamp", "medamp", "minamp", "maxamp", "rangeamp", "prom"  , "Tind"  , "startind", "start"   , "Acwt"  , "Q"     , "g"     , "Afit"  , "cfit"  , "rsquare", "dfe"   , "adjrsquare", "rmse"  , "goodness", "windowstartTime", "channelCount", "amp_HV_NPB_HHE", "amp_HV_NPB_HHN", "amp_HV_NPB_HHZ", "amp_HV_SRM_HHE", "amp_HV_SRM_HHN", "amp_HV_SRM_HHZ", "amp_HV_NPT_HHE", "amp_HV_NPT_HHN", "amp_HV_NPT_HHZ", "amp_HV_OBL_HHE", "amp_HV_OBL_HHN", "amp_HV_OBL_HHZ", "amp_HV_WRM_HHE", "amp_HV_WRM_HHN", "amp_HV_WRM_HHZ", "amp_HV_SDH_HHE", "amp_HV_SDH_HHN", "amp_HV_SDH_HHZ", "amp_HV_UWE_HHE", "amp_HV_UWE_HHN", "amp_HV_UWE_HHZ", "amp_HV_UWB_HHE", "amp_HV_UWB_HHN", "amp_HV_UWB_HHZ", "amp_HV_SBL_HHE", "amp_HV_SBL_HHN", "amp_HV_SBL_HHZ", "amp_HV_KKO_HHE", "amp_HV_KKO_HHN", "amp_HV_KKO_HHZ", "amp_HV_RIMD_HHE", "amp_HV_RIMD_HHN", "amp_HV_RIMD_HHZ", "phase_HV_NPB_HHE", "phase_HV_NPB_HHN", "phase_HV_NPB_HHZ", "phase_HV_SRM_HHE", "phase_HV_SRM_HHN", "phase_HV_SRM_HHZ", "phase_HV_NPT_HHE", "phase_HV_NPT_HHN", "phase_HV_NPT_HHZ", "phase_HV_OBL_HHE", "phase_HV_OBL_HHN", "phase_HV_OBL_HHZ", "phase_HV_WRM_HHE", "phase_HV_WRM_HHN", "phase_HV_WRM_HHZ", "phase_HV_SDH_HHE", "phase_HV_SDH_HHN", "phase_HV_SDH_HHZ", "phase_HV_UWE_HHE", "phase_HV_UWE_HHN", "phase_HV_UWE_HHZ", "phase_HV_UWB_HHE", "phase_HV_UWB_HHN", "phase_HV_UWB_HHZ", "phase_HV_SBL_HHE", "phase_HV_SBL_HHN", "phase_HV_SBL_HHZ", "phase_HV_KKO_HHE", "phase_HV_KKO_HHN", "phase_HV_KKO_HHZ", "phase_HV_RIMD_HHE", "phase_HV_RIMD_HHN", "phase_HV_RIMD_HHZ", "errphase_HV_NPB_HHE", "errphase_HV_NPB_HHN", "errphase_HV_NPB_HHZ", "errphase_HV_SRM_HHE", "errphase_HV_SRM_HHN", "errphase_HV_SRM_HHZ", "errphase_HV_NPT_HHE", "errphase_HV_NPT_HHN", "errphase_HV_NPT_HHZ", "errphase_HV_OBL_HHE", "errphase_HV_OBL_HHN", "errphase_HV_OBL_HHZ", "errphase_HV_WRM_HHE", "errphase_HV_WRM_HHN", "errphase_HV_WRM_HHZ", "errphase_HV_SDH_HHE", "errphase_HV_SDH_HHN", "errphase_HV_SDH_HHZ", "errphase_HV_UWE_HHE", "errphase_HV_UWE_HHN", "errphase_HV_UWE_HHZ", "errphase_HV_UWB_HHE", "errphase_HV_UWB_HHN", "errphase_HV_UWB_HHZ", "errphase_HV_SBL_HHE", "errphase_HV_SBL_HHN", "errphase_HV_SBL_HHZ", "errphase_HV_KKO_HHE", "errphase_HV_KKO_HHN", "errphase_HV_KKO_HHZ", "errphase_HV_RIMD_HHE", "errphase_HV_RIMD_HHN", "errphase_HV_RIMD_HHZ", "mean_errphase", "mean_min50_errphase", "fm_start", "fm_A_above_LTA", "fm_STALTA", "fm_Afrac", "fm_HV_NPB_HHE", "fm_HV_NPB_HHN", "fm_HV_NPB_HHZ", "fm_HV_SRM_HHE", "fm_HV_SRM_HHN", "fm_HV_SRM_HHZ", "fm_HV_NPT_HHE", "fm_HV_NPT_HHN", "fm_HV_NPT_HHZ", "fm_HV_OBL_HHE", "fm_HV_OBL_HHN", "fm_HV_OBL_HHZ", "fm_HV_WRM_HHE", "fm_HV_WRM_HHN", "fm_HV_WRM_HHZ", "fm_HV_SDH_HHE", "fm_HV_SDH_HHN", "fm_HV_SDH_HHZ", "fm_HV_UWE_HHE", "fm_HV_UWE_HHN", "fm_HV_UWE_HHZ", "fm_HV_UWB_HHE", "fm_HV_UWB_HHN", "fm_HV_UWB_HHZ", "fm_HV_SBL_HHE", "fm_HV_SBL_HHN", "fm_HV_SBL_HHZ", "fm_HV_KKO_HHE", "fm_HV_KKO_HHN", "fm_HV_KKO_HHZ", "fm_HV_RIMD_HHE", "fm_HV_RIMD_HHN", "fm_HV_RIMD_HHZ"];
% opts.VariableTypes = ["double", "double", "double" , "double", "double", "double", "double"  , "double", "double", "double"  , "datetime", "double", "double", "double", "double", "double", "double" , "double", "double"    , "double", "double"  , "datetime"       , "double"      , "double"        , "double"        , "double"        , "double"        , "double"        , "double"        , "double"        , "double"        , "double"        , "double"        , "double"        , "double"        , "double"        , "double"        , "double"        , "double"        , "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "string", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double"];

opts = delimitedTextImportOptions("NumVariables", 137);

% Specify range and delimiter
opts.DataLines = [2, Inf];
opts.Delimiter = ",";

% Specify column names and types
opts.VariableNames = ["T", "stdamp", "meanamp", "medamp", "minamp", "maxamp", "rangeamp", "prom", "Tind", "startind", "start", "Acwt", "Q", "g", "Afit", "cfit", "rsquare", "dfe", "adjrsquare", "rmse", "goodness", "windowstartTime", "channelCount", "amp_HV_NPT_HHE", "amp_HV_NPT_HHN", "amp_HV_NPT_HHZ", "amp_HV_OBL_HHE", "amp_HV_OBL_HHN", "amp_HV_OBL_HHZ", "amp_HV_WRM_HHE", "amp_HV_WRM_HHN", "amp_HV_WRM_HHZ", "amp_HV_SDH_HHE", "amp_HV_SDH_HHN", "amp_HV_SDH_HHZ", "amp_HV_UWE_HHE", "amp_HV_UWE_HHN", "amp_HV_UWE_HHZ", "amp_HV_UWB_HHE", "amp_HV_UWB_HHN", "amp_HV_UWB_HHZ", "amp_HV_SBL_HHE", "amp_HV_SBL_HHN", "amp_HV_SBL_HHZ", "amp_HV_KKO_HHE", "amp_HV_KKO_HHN", "amp_HV_KKO_HHZ", "amp_HV_RIMD_HHE", "amp_HV_RIMD_HHN", "amp_HV_RIMD_HHZ", "phase_HV_NPT_HHE", "phase_HV_NPT_HHN", "phase_HV_NPT_HHZ", "phase_HV_OBL_HHE", "phase_HV_OBL_HHN", "phase_HV_OBL_HHZ", "phase_HV_WRM_HHE", "phase_HV_WRM_HHN", "phase_HV_WRM_HHZ", "phase_HV_SDH_HHE", "phase_HV_SDH_HHN", "phase_HV_SDH_HHZ", "phase_HV_UWE_HHE", "phase_HV_UWE_HHN", "phase_HV_UWE_HHZ", "phase_HV_UWB_HHE", "phase_HV_UWB_HHN", "phase_HV_UWB_HHZ", "phase_HV_SBL_HHE", "phase_HV_SBL_HHN", "phase_HV_SBL_HHZ", "phase_HV_KKO_HHE", "phase_HV_KKO_HHN", "phase_HV_KKO_HHZ", "phase_HV_RIMD_HHE", "phase_HV_RIMD_HHN", "phase_HV_RIMD_HHZ", "errphase_HV_NPT_HHE", "errphase_HV_NPT_HHN", "errphase_HV_NPT_HHZ", "errphase_HV_OBL_HHE", "errphase_HV_OBL_HHN", "errphase_HV_OBL_HHZ", "errphase_HV_WRM_HHE", "errphase_HV_WRM_HHN", "errphase_HV_WRM_HHZ", "errphase_HV_SDH_HHE", "errphase_HV_SDH_HHN", "errphase_HV_SDH_HHZ", "errphase_HV_UWE_HHE", "errphase_HV_UWE_HHN", "errphase_HV_UWE_HHZ", "errphase_HV_UWB_HHE", "errphase_HV_UWB_HHN", "errphase_HV_UWB_HHZ", "errphase_HV_SBL_HHE", "errphase_HV_SBL_HHN", "errphase_HV_SBL_HHZ", "errphase_HV_KKO_HHE", "errphase_HV_KKO_HHN", "errphase_HV_KKO_HHZ", "errphase_HV_RIMD_HHE", "errphase_HV_RIMD_HHN", "errphase_HV_RIMD_HHZ", "mean_errphase", "mean_min50_errphase", "fm_start", "fm_A_above_LTA", "fm_STALTA", "fm_Afrac", "fm_HV_NPT_HHE", "fm_HV_NPT_HHN", "fm_HV_NPT_HHZ", "fm_HV_OBL_HHE", "fm_HV_OBL_HHN", "fm_HV_OBL_HHZ", "fm_HV_WRM_HHE", "fm_HV_WRM_HHN", "fm_HV_WRM_HHZ", "fm_HV_SDH_HHE", "fm_HV_SDH_HHN", "fm_HV_SDH_HHZ", "fm_HV_UWE_HHE", "fm_HV_UWE_HHN", "fm_HV_UWE_HHZ", "fm_HV_UWB_HHE", "fm_HV_UWB_HHN", "fm_HV_UWB_HHZ", "fm_HV_SBL_HHE", "fm_HV_SBL_HHN", "fm_HV_SBL_HHZ", "fm_HV_KKO_HHE", "fm_HV_KKO_HHN", "fm_HV_KKO_HHZ", "fm_HV_RIMD_HHE", "fm_HV_RIMD_HHN", "fm_HV_RIMD_HHZ"];
opts.VariableTypes = ["double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "datetime", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "datetime", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "datetime", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double", "double"];


% Specify file level properties
opts.ExtraColumnsRule = "ignore";
opts.EmptyLineRule = "read";

% Specify variable properties
opts = setvaropts(opts, "start", "InputFormat", "dd-MMM-yyyy HH:mm:ss");
opts = setvaropts(opts, "windowstartTime", "InputFormat", "dd-MMM-yyyy HH:mm:ss");
opts = setvaropts(opts, "fm_start", "InputFormat", "dd-MMM-yyyy HH:mm:ss");
opts = setvaropts(opts, "T", "TrimNonNumeric", true);
opts = setvaropts(opts, "T", "ThousandsSeparator", ",");


for batchin = 1:batchcount
    if exist(strcat(foldername,'/',num2str(batchin),'.csv'),'file') == 2
        TQ_tab_local = readtable(strcat(foldername,'/',num2str(batchin),'.csv'),...
            opts);
%         TQ_tab_local.fm_start = datetime(TQ_tab_local.fm_start,'InputFormat',"dd-MMM-yyyy HH:mm:ss");
        if ~exist('TQ_tab','var')
            TQ_tab = TQ_tab_local;
        else
            TQ_tab = cat(1,TQ_tab,TQ_tab_local);
        end
    end
end
warning('on','all')
%save
writetable(TQ_tab, strcat(foldername,'/total','.csv'));